import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import re # Import library for string manipulation
import string
import time # Import libraries in case we need to make the requests seem less robotic
import random
# Use Selenium to add options to the browser, for conditional waiting, scrolling and more.
# Most importantly, we use it because the website has dynamic content for which JavaScript is utilised.
# With Selenium we can read that content and work with it alongside the normal html code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import threading
from threading import Thread
import queue
import copy
import json

# Choose your preferred inputs below. These are for LSE:
postcode = "WC2A 2AE"
area = "Westminster"
county = "Greater London"
radius = "4"
prices_min = "100"
prices_max = "" # We do not add limits on the price as we want to analyse the price. If users want, add " &prices_max=x" to the url.
bed_max = "6"
accept_students = "true"

# Manipulate to get desired formats
outcode = postcode.split()[0]; incode = postcode.split()[-1] # Split postcode into outcode and incode
full_area = re.sub(r'\s+', "-", f'{area} {county}') # Join area and county and substitute spaces with "-"
area_url = re.sub(r'\s+', "%20", area); county_url = re.sub(r'\s+', "%20", county) # Substitute spaces with '%20' in area and county.

# Set up Chrome options to make running the url headless, i.e. run the commands without actually opening a browser window.
# This way we speed the process as there's no need for all the visual representation:
chrome_ops = Options()
chrome_ops.add_argument("--headless")

# Choose chrome as our browser and add the headless option:
browser = webdriver.Chrome(options=chrome_ops)

# Define the url with the parameters above and get it:
url_1 = f'https://www.openrent.co.uk/properties-to-rent/{outcode.lower()}-{incode.lower()}-{full_area.lower()}?term={outcode}%20{incode}%20{area_url},%20{county_url}&area={radius}&prices_min={prices_min}&bedrooms_max={bed_max}&acceptStudents={accept_students}'
browser.get(url_1) # Open the url in the browser

# Wait for the whole body to become visible:
wait = WebDriverWait(browser, 30) # Set a maximum waiting time of 30 seconds
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "body")))

# Scroll until the height 1 second after we scroll is the same as the height before the scroll.
# This shows us when there are no more properties to load.
# Make the loop infinite as the variables of the condition have to be defined inside of the loop and then "break" it:
while "We are" != "ST115 superstars":
    old_h = browser.execute_script("return document.body.scrollHeight") # Find the height before the scroll

    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") # Scroll to the bottom
    time.sleep(1)
    
    new_h = browser.execute_script("return document.body.scrollHeight") # Find the height after the scroll and the load of 1 sec
    if new_h == old_h:
        break

# Get the page source after we have finished scrolling and it has fully loaded:
html_code = browser.page_source

# Close the driver to avoid it running in the background:
browser.quit()

# Make the regular soup out of the html code:
main_soup = BeautifulSoup(html_code, 'lxml')

# Save the code to a file:
with open("main_property_page_code.html", 'w', encoding='utf-8') as file:
    file.write(html_code)
#print(main_soup)

# Find the number of all properties we analyse + the number of properties that are no longer available:
all_properties = len(main_soup.find_all("a", class_ = "pli clearfix")) + len(main_soup.find_all("a", class_ = "pli l-a clearfix"))
# Get the number of total properties stated at the top of the page:
number_of_properties_on_page = int(main_soup.find("div", class_="contentPane__content transparent").find("div", id='top-detail-bar').find("div", class_='search-detail').find("span", class_='filter-info').find("span").get_text().strip())

print(f"Our approach is correct - \033[1m{all_properties==number_of_properties_on_page}\033[0m - and the total number of properties is \033[1m{all_properties}\033[0m.")

Our approach is correct - True - and the total number of properties is 787.

# Find all sections for different properties on the main page:
all_available_props = main_soup.find_all("a", class_ = "pli clearfix")

def get_distance(prop):
    dist = prop.find("div", class_ = "price-location clearfix").find("div", class_="ltc pl-title").find("h2").get_text().strip()
    return dist
    
def get_price(prop):
    price = prop.find("div", class_="price-location clearfix").find("div", class_="pim pl-title").find("h2").get_text().strip()
    return price
    
def get_type_and_outcode(prop):
    title = prop.find("div", class_="location-description").find("span").get_text().strip()
    title_split = title.split(',') # Make a list with the components in the title
    prop_type = title_split[0].strip().lower() # Take the first component (property type) and make it lowercase
    outcode = title_split[-1].strip() # Take the outcode (first part of postcode)
    return [prop_type, outcode]
    
props = {} # Create an empty list to populate with the data
# Iterate through them and get the info we require:
for prop in all_available_props:
    prop_id = prop["href"].strip("/")
    prop_url = "https://www.openrent.co.uk/" + prop_id
    type_and_outcode = get_type_and_outcode(prop)
    # Put a list of the property url, price and distance in the dictionary, use the id for a key:
    props[prop_id] = [prop_url, get_price(prop), get_distance(prop), type_and_outcode[0], type_and_outcode[-1]]
    
#display only first 3 listings from the resulting dictionary:
display({k:props[k] for k in list(props)[:3]})

{'2016944': ['https://www.openrent.co.uk/2016944',
  '£7,800 per month',
  '0.17  km',
  '3 bed flat',
  'WC2A'],
 '2043123': ['https://www.openrent.co.uk/2043123',
  '£2,396 per month',
  '0.31  km',
  'studio flat',
  'WC2B'],
 '2016841': ['https://www.openrent.co.uk/2016841',
  '£5,633 per month',
  '0.35  km',
  '2 bed flat',
  'WC2R']}

print(f'We are currently analysing \033[1m{len(props)}\033[0m properties.') # Note these are the still-available ones.

We are currently analysing 753 properties.

%%prun

props_i = copy.deepcopy(props)

class TaskQueue(queue.Queue):

    # Inherit from the Queue.Queue class to get access to the get, put methods and the queue behaviour:
    def __init__(self, num_workers=1): # Set some default number of workers which we will change in tests()
        queue.Queue.__init__(self)
        self.num_workers = num_workers
        self.start_workers()

    # Store the task, args, kwargs as tuples in the queue.
    # Note that *args allows passing variable number of arguments and **kwargs allows named arguments
    def add_task(self, task, *args, **kwargs):
        args = args or ()
        kwargs = kwargs or {}
        self.put((task, args, kwargs))

    # Create threads for each of the workers and fire them off in background mode:
    def start_workers(self):
        for i in range(self.num_workers):
            t = Thread(target=self.worker)
            t.daemon = True
            t.start()

    # Add the actual worker code.
    # Note that the worker basically just gets the first task in the queue and runs it.
    def worker(self):
        while True:
            item, args, kwargs = self.get()
            item(*args, **kwargs)  
            self.task_done()

# To test the queue but also run the whole threading:
def tests():
    # Define the actual task:
    def task(props_i, *args, **kwargs):
        # Open the chrome
        chrome_ops = Options()
        chrome_ops.add_argument("--headless")
        browser = webdriver.Chrome(options=chrome_ops)
        url = str(args[0]) # Get URL
        #print("open br")
        try:
            browser.get(url)
            # Find the static google maps picture and click on it:
            browser.find_element(By.XPATH, "//img[@id='staticGoogleMap']").click()
            #print("click on element")

            # Wait until the interactive map appear and set max waiting time:
            WebDriverWait(browser, 3).until(EC.presence_of_element_located((By.XPATH, "//div[@id='googlemapandstreetview']")))
            code = browser.page_source # Get the html code
            soup = BeautifulSoup(code, 'lxml') # Turn it into a soup
            #print("element located successfully")
            
            # Get the overview table:
            table = soup.find("table", class_="table table-striped intro-stats")
            cells = table.find_all("td") # Get each cell with data
            #print("found table")
            
            # Make 2 lists - 1 with the type of data - bedroom/bathroom/max tenants and one with the values.
            # Note that we do not need the location cell, which is the last one out of 4.
            data_var = [cell.find("span").get_text(strip=True).strip(":") for cell in cells[:3]] # Strip spaces and ":"
            value = [cell.find("strong").get_text(strip = True) for cell in cells[:3]]
            bbt = dict(zip(data_var, value)) # Make a dictionary out of it

            # Find the coordinates by searching for a pattern in the code string. Note we use code and not soup!
            # Note that there is a line of code that defines a variable and inputs the coordinates always:
            # The line in the JavaScript code is "var latlng = new L.LatLng(latitude, longitude);"
            var_latlng = r'new L\.LatLng\(([^,]+),([^)]+)\);'
            match = re.search(var_latlng, code) # Find the pattern in the code string
            lat = float(match.group(1))
            lng = float(match.group(2))

            # Get the id from the url and use it to reference the dictionary and append the corresponding list with our bbt dict.
            prop_id = url[len("https://www.openrent.co.uk/"):]
            props_i[prop_id].append(bbt)
            props_i[prop_id].append([lat, lng])
            #print("dict appended")
        
        except Exception as e:
            # Handle the exception gracefully
            print("An exception occurred:", e)
            # Perform actions or logging specific to the exception

            # Increment a counter to keep track of the number of occurrences
            # You can define the counter variable outside the task function
            global exception_count
            exception_count += 1
        
        finally:
            browser.quit() # Close the browser
            #print("close br")
            
    # CHOOSE THE NUMBER OF WORKERS:
    q = TaskQueue(num_workers=2)

    for prop in list(props_i.values()): # Scrape the list from the dictionary for urls and keep adding tasks
        url = prop[0]
        q.add_task(task, props_i, url)

    q.join() # Block until all tasks are done
    # Save the dictioinary in a JSON file so that we have the data saved and we do not have to rerun every time:
    with open("data/private_property_info.json", "w") as json_file:
        json.dump(props_i, json_file)
    print("All done")
    display({k:props_i[k] for k in list(props_i)[:3]})

if __name__ == "__main__":
    global exception_count
    exception_count = 0
    tests() # Here we call the tests function which runs everything if the script is executed directly
    print(f"There has been an exception for {exception_count} of the properties")

All done

{'2016944': ['https://www.openrent.co.uk/2016944',
  '£7,800 per month',
  '0.17  km',
  '3 bed flat',
  'WC2A',
  {'Bedrooms': '3', 'Bathrooms': '2', 'Max Tenants': '5'},
  [51.51498, -0.1148634]],
 '2043123': ['https://www.openrent.co.uk/2043123',
  '£2,396 per month',
  '0.31  km',
  'studio flat',
  'WC2B',
  {'Bedrooms': '1', 'Bathrooms': '1', 'Max Tenants': '1'},
  [51.514355, -0.1211351]],
 '2016841': ['https://www.openrent.co.uk/2016841',
  '£5,633 per month',
  '0.35  km',
  '2 bed flat',
  'WC2R',
  {'Bedrooms': '2', 'Bathrooms': '2', 'Max Tenants': '4'},
  [51.511524, -0.1136339]]}

There has been an exception for 0 of the properties

#create a function for getting a crimes number from given coordinates and month
def coord_crime_num_month(lat,long,month):
    
    #get all crime data from coordinates and every month, using police.uk API
    monthcrimecount = len(requests.get(f'https://data.police.uk/api/crimes-street/all-crime?lat={lat}&lng={long}&date={month}').json())
        
    #return the total number of crimes that data is given for
    return(monthcrimecount)

#create a function for getting a crimes number from given coordinates and year
def coord_crime_num_year(lat,long,year):
    
    #get all crime data from coordinates and every month, using police.uk API, and sums them for total crimes in the year
    yearcrimecount = 0
    for month in range(1,13):
        yearcrimecount += len(requests.get(f'https://data.police.uk/api/crimes-street/all-crime?lat={lat}&lng={long}&date={year}-{month}').json())
        
    #return the total number of crimes that data is given for
    return(yearcrimecount)

#open previous json file 
with open('data/private_property_info.json') as f:
    props_c = json.load(f)

#iterate through each property and extract coordinates
for prop in props_c: 
    coords = props_c[prop][-1]
    
    #define lat and long from coords
    lat = coords[0]
    long = coords[1]
    
    #run crime number API function using given coords, in most recent month available
    crime_num = coord_crime_num_month(lat,long,'2024-01')
    
    #append area crimerate to end of property info list
    props_c[prop].append(crime_num)

#save dictionary in a JSON file so that we have the data saved and we do not have to rerun every time
with open('data/private_property_info_with_crimes_number.json', 'w') as json_file:
    json.dump(props_c, json_file)
    
print('All done')

All done

#create empty list to fill with URLs for each relevant hall.
url_list = []

#there are two pages of results from the query we submitted for the URL, so will need to iterate through both pages' URLs
for index in range(1,3):
    
    #uses f-string for url to substitute in the page number, then retrieves the page's 'soup'
    url = f'https://www.lse.ac.uk/student-life/accommodation/search-accommodation?collection=lse-accommodation&pageIndex={index}&roomType=451ee418-b4c2-4727-8ea8-735dd74f33ce&sort=metaavailability%20%20&studentType=95ff59b3-9e8a-4ad6-83bd-f4c3357b2def'
    r = requests.get(url)
    soup = BeautifulSoup(r.content)
    
    #iterate through each 'card', finds next 'a' element's href, which is the URL of the hall's webpage, then adds to the list
    for item in soup.find_all('h2',attrs={'class':"card__title"}):
        url = item.find_next('a')['href'].strip('.aspx')
        url_list.append(url)

#create empty pd dataframe to fill with all possibly useful info for each hall
halls_i = pd.DataFrame(columns=['hall','(avg) cost per week','contract length','(avg) contract cost','distance from LSE','crimes number','outcode'])

#iterate through each url in the list
for url in url_list:
    
    #retrieve soup for the url
    r = requests.get(url)
    soup = BeautifulSoup(r.content)
    
    #find name of hall from herobanner title
    name = soup.find('h1',attrs={'class':'heroBanner__title'}).get_text()
    
    #find distance from LSE from herobanner content
    distance_text = soup.find('div',attrs={'class':'accommKeyDetails__dist'}).get_text()
    distance = distance_text.strip('km').strip().split()[-1]
    
    #finds piece of text with contract length, the third word of which is always the contract length itself
    contract_text = soup.find('p',attrs={'class':'accommKeyDetails__contract'}).get_text()
    contract_length = int(contract_text.split()[2])
    
    #finds location of where the single room information is
    #had to use two checks as some are listed as 'Single room', others as 'Single Room '
    single_location = soup.find('h2',attrs={'class':'roomlist__title'},string='Single room') 
    if single_location == None:
        single_location = soup.find('h2',attrs={'class':'roomlist__title'},string='Single room ')
    
    #finds next 'room at a glance price' element, and extracts the text from the element
    price_info = single_location.find_next('p',attrs={'class':'roomataGlance__price'})
    price_info_text = str(price_info.get_text()).split()
    
    #some have a single price and some contain a range of prices
    #single prices have a length of 3, with the second word being the price per week
    if len(price_info_text)==3:
        price = price_info_text[1].strip('£')
        
    #ranges of prices have a length of 5, with the lower in the second and upper in the fourth
    #I took a mean of the upper and lower bound as a representative single price
    if len(price_info_text)==5:
        price = (float(price_info_text[1].strip('£'))+float(price_info_text[3].strip('£')))/2
    
    #calculates contract cost by multiplying contract length and price
    contract_cost = float(contract_length) * float(price)
    
    #find postcode so that a crimes number can be calculated
    address_text = soup.find('h2',attrs={'class':'_mce_tagged_br'}).find_next('p').get_text()
    postcode = address_text.split('London,')[1].strip()
    
    #from testing, I have found that the webpage for garden halls has a typo in the postcode - it reads WC1H 9EB, but should read WC1H 9EN.
    #we have created a line of code to correct this error.
    if postcode == 'WC1H 9EB':
        postcode = 'WC1H 9EN'
        
    #we will need outcode so that we can compare rough locations with rental properties
    outcode = postcode.split()[0]
    
    #use postcodes.io API to find coordinates for the given postcode
    api_response = requests.get(f'https://api.postcodes.io/postcodes/{postcode}').json()
    lat = api_response["result"]["latitude"]
    long = api_response["result"]["longitude"]
    
    #input coordinates into crimes number API function to return a crimes number
    crime_num = coord_crime_num_month(lat,long,'2024-01')
    
    #adds new row to the halls_i dataframe
    halls_i.loc[len(halls_i.index)] = [name,price,contract_length,contract_cost,distance,crime_num,outcode]

#save dataframe as a csv file so that it can be accessed without running code again
halls_i.to_csv('data/lse_accomm_info.csv', sep=',', index=False, encoding='utf-8')

#create empty pd dataframe to house all data
all_info = pd.DataFrame(columns=['Name','Outcode','Price per month','Price per bedroom per month','Price per bedroom per year','No. bedrooms','Distance from LSE','Crime No.','LSE accommodation'])

#open properties with crimes number json file as props_p
with open('data/private_property_info_with_crimes_number.json') as f:
    props_p = json.load(f)
    
#iterate through each property, gathering datapoints to be put into pd dataframe
for prop in props_p:
    name = props_p[prop][3]
    
    outcode = props_p[prop][4]
    
    #need to strip text out of price per month information, to obtain a number
    ppm_text = props_p[prop][1]
    ppm = int(ppm_text.strip('£').strip(' per month').replace(',',''))
    
    no_bedrooms = int(props_p[prop][5]['Bedrooms'])
    
    #create if statement to define whether house/flat is shared
    shared_house = False
    name_words = name.lower().split()
    if 'shared' in name_words:
        ppm_per_room = ppm
    else:
        ppm_per_room = ppm/no_bedrooms
        
    ppy_per_room = 12*ppm_per_room
        
    dist_lse = float(props_p[prop][2].split()[0])
    
    crime_num =  props_p[prop][7]
    
    #signal that property is not an LSE hall
    lse_accomm = False
    
    #append info to pd dataframe
    all_info.loc[len(all_info.index)] = [name,outcode,ppm,ppm_per_room,ppy_per_room,no_bedrooms,dist_lse,crime_num,lse_accomm]
    
all_info.head()

#open csv file as a pd dataframe
halls_p = pd.read_csv('data/lse_accomm_info.csv')
halls_p

#iterate through the content of each row of the halls dataframe
for index,row in halls_p.iterrows():
    
    #label each piece of content from row by the way it will be addes to all_info dataframe
    
    name = row['hall']
    outcode = row['outcode']
    ppy = int(row['(avg) contract cost'])
    dist_lse = row['distance from LSE']
    crime_num = row['crimes number']
    
    #calculate ppm as price per week multiplied by avg weeks per month
    ppm = int(row['(avg) cost per week']*(52/12))
    ppm_per_bedroom = ppm
    
    #all of these rows are for 1 bedroom, and are a part of lse halls 
    no_bedrooms = 1
    lse_accomm = True
    
    #append row to end of all_info dataframe
    all_info.loc[len(all_info.index)] = [name,outcode,ppm,ppm_per_bedroom,ppy,no_bedrooms,dist_lse,crime_num,lse_accomm]

#As can be seen below, the LSE accommodations are added to the bottom of the pandas df:
all_info[-13:]

all_info.to_csv('data/private_and_lse_final_info.csv', sep=',', index=False, encoding='utf-8')

# Import extra libraries for visualisation and analysis:
import seaborn as sns
import plotly.express as px
import statsmodels.formula.api as sm
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')

all_i = pd.read_csv('data/private_and_lse_final_info.csv')
display(all_i)

display(all_i.info())
display(all_i.describe(include="all").round(2)) # Round for readability
display(all_i.isna().sum())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 764 entries, 0 to 763
Data columns (total 9 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   Name                         764 non-null    object 
 1   Outcode                      764 non-null    object 
 2   Price per month              764 non-null    int64  
 3   Price per bedroom per month  764 non-null    float64
 4   Price per bedroom per year   764 non-null    float64
 5   No. bedrooms                 764 non-null    int64  
 6   Distance from LSE            764 non-null    float64
 7   Crime No.                    764 non-null    int64  
 8   LSE accommodation            764 non-null    bool   
dtypes: bool(1), float64(3), int64(3), object(2)
memory usage: 48.6+ KB

None

Name                           0
Outcode                        0
Price per month                0
Price per bedroom per month    0
Price per bedroom per year     0
No. bedrooms                   0
Distance from LSE              0
Crime No.                      0
LSE accommodation              0
dtype: int64

# Open the initial json file with the long dictionary to be able to see the links and all other info: 
with open('data/private_property_info.json') as f:
    props_check = json.load(f)

for prop in props_check:
    if int(props_check[prop][-2]["Bedrooms"]) > 6:
        display(props_check[prop])

['https://www.openrent.co.uk/2022187',
 '£1,296 per month',
 '1.61  km',
 'room in a shared flat',
 'SE1',
 {'Bedrooms': '7', 'Bathrooms': '7', 'Max Tenants': '1'},
 [51.501884, -0.1038921]]

['https://www.openrent.co.uk/2018779',
 '£900 per month',
 '3.78  km',
 'room in a shared house',
 'NW8',
 {'Bedrooms': '7', 'Bathrooms': '5', 'Max Tenants': '1'},
 [51.525555, -0.1680249]]

all_i = all_i[all_i["No. bedrooms"] <= 6]
all_i.describe().round(2)

# Make the display panel fit the whole graph:
from IPython.display import display, HTML
display(HTML("<style>.output { height: auto !important; }</style>"))

# Find the correlations of the columns corresponding to price per bedroom per month, crimes, no. of bedrooms and distance from LSE.
# Exclude LSE accommodations for now.
corr_matrix = all_i.loc[all_i['LSE accommodation']==False, ['Price per bedroom per month', "No. bedrooms", "Distance from LSE", "Crime No."]].corr()
# Visualise the matrix via a heatmap - different shade of the colour depending of the significance of the correlation.

plt.figure(figsize=(6, 5))

# Create the heatmap and add the values and round them to 2 decimal places.
# Also center around 0 so that values of negative correlations and positive ones are shades of different colours:
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt=".2f", center=0, annot_kws = {'size': 8, 'fontweight': "bold"}, vmin=-1) # The last parameter is so that we set the min value of the bar.
plt.xticks(rotation = 30, ha = "right", fontsize = 10)
plt.yticks(rotation = 0, fontsize = 10)
plt.title('Correlation Matrix of the Data', fontsize = 10, fontweight='bold')
plt.tight_layout()

plt.show()

plt.figure(figsize=(8, 6))
# We plot only the listed properties on OpenRent, so we exclude the LSE accommodations here:
ax = sns.boxplot(all_i[all_i['LSE accommodation']==False],x='No. bedrooms',y='Price per bedroom per month',palette = 'tab10')
ax.set_title('Distribution of price per bedroom per month by number of bedrooms in property', fontdict={'fontsize': 12, 'fontweight': 'bold'})
ax.set_ylabel("Price per bedroom per month (£)", fontsize = 12)
ax.set_xlabel("No. bedrooms", fontsize = 12)
ax.grid(axis='y', linestyle='--', alpha=0.5)
ax.plot()

[]

all_i[['No. bedrooms', 'Price per bedroom per month']].groupby('No. bedrooms').describe().round(2)

fig, ax = plt.subplots(1, 2, figsize=(16, 8))

ax_1 = sns.barplot(data = all_i[all_i['LSE accommodation']==False]['No. bedrooms'].value_counts().reset_index(),x='No. bedrooms',y='count', ax = ax[0], palette = 'tab10')
for i in ax_1.containers:
    ax_1.bar_label(i,)
ax_1.set_ylabel('No. available properties', fontsize = 12)
ax_1.set_xlabel("No. bedrooms", fontsize = 12)
ax_1.set_title('Number of Available Properties by No. of Bedrooms', fontdict={'fontsize': 14, 'fontweight': 'bold'})

ax[1].axis("equal")
ax[1].set_title('Percentage (%) of Available Properties by No. of Bedrooms', fontsize = 14, fontweight = 'bold')

labels = all_i[all_i['LSE accommodation']==False]["No. bedrooms"].value_counts().index
numbers = all_i[all_i['LSE accommodation']==False]["No. bedrooms"].value_counts().values

sections, nums, percentages = ax[1].pie(numbers, autopct = '%1.1f%%')
ax[1].legend(labels, loc='upper right', fontsize = 12)

for percentage in percentages:
    percentage.set_fontsize(12)
    percentage.set_fontweight('bold')

plt.show()

plt.figure(figsize=(10, 6))

affordable_props = all_i[all_i['LSE accommodation']==False][all_i['Price per bedroom per month']<1400][['Outcode','Price per bedroom per month']].groupby('Outcode').count().reset_index().sort_values('Price per bedroom per month')
affordable_props_crimerate = all_i[['Outcode','Crime No.']].groupby('Outcode').mean()
affordable_props_combined = affordable_props.join(affordable_props_crimerate, how='left', on='Outcode').rename({'Price per bedroom per month':'Number of Available Properties below £1400 per Bedroom','Crime No.':'Avg Crime No. in Outcode'},axis=1)

ax = sns.barplot(affordable_props_combined,x='Outcode',y='Number of Available Properties below £1400 per Bedroom',hue = 'Avg Crime No. in Outcode')

ax.set_xticklabels(list(affordable_props_combined['Outcode']), rotation=90)
ax.set_xlabel("Outcode", fontsize = 12)
ax.set_ylabel("Number of Properties below £1400 per Bedroom", fontsize = 12)
ax.set_title("Number of Affordable Properties by Area", fontsize = 14, fontweight = 'bold')
ax.grid(axis='y', linestyle='--', alpha=0.5)
ax.plot()

[]

affordable_props_combined.sort_values(by='Number of Available Properties below £1400 per Bedroom', ascending=False).reset_index().round(2)[:5]

ols_df = all_i[['Price per bedroom per month','No. bedrooms','Distance from LSE','Crime No.','LSE accommodation']]
ols_df['LSE accommodation'] = ols_df['LSE accommodation'].astype('int')
ols_df = ols_df.rename(columns={'Price per bedroom per month':'price_per_bedroom_per_month','No. bedrooms':'no_bedrooms','Distance from LSE':'distance_from_LSE','Crime No.':'crime_no','LSE accommodation':'LSE_accommodation'})

result = sm.ols(formula='price_per_bedroom_per_month ~ no_bedrooms + distance_from_LSE + crime_no + LSE_accommodation', 
                data=ols_df).fit(cov_type='HC0') # Use robust standard errors
print(result.summary())

                                 OLS Regression Results                                
=======================================================================================
Dep. Variable:     price_per_bedroom_per_month   R-squared:                       0.308
Model:                                     OLS   Adj. R-squared:                  0.304
Method:                          Least Squares   F-statistic:                     132.3
Date:                         Thu, 02 May 2024   Prob (F-statistic):           1.15e-85
Time:                                 10:33:11   Log-Likelihood:                -6086.8
No. Observations:                          762   AIC:                         1.218e+04
Df Residuals:                              757   BIC:                         1.221e+04
Df Model:                                    4                                         
Covariance Type:                           HC0                                         
=====================================================================================
                        coef    std err          z      P>|z|      [0.025      0.975]
-------------------------------------------------------------------------------------
Intercept          2684.0227    195.019     13.763      0.000    2301.793    3066.252
no_bedrooms        -376.5173     20.821    -18.084      0.000    -417.326    -335.709
distance_from_LSE   -62.7628     44.895     -1.398      0.162    -150.756      25.230
crime_no              0.0400      0.036      1.114      0.265      -0.030       0.110
LSE_accommodation -1241.6619     85.828    -14.467      0.000   -1409.882   -1073.441
==============================================================================
Omnibus:                      427.044   Durbin-Watson:                   1.776
Prob(Omnibus):                  0.000   Jarque-Bera (JB):             5193.308
Skew:                           2.273   Prob(JB):                         0.00
Kurtosis:                      14.954   Cond. No.                     2.62e+04
==============================================================================

Notes:
[1] Standard Errors are heteroscedasticity robust (HC0)
[2] The condition number is large, 2.62e+04. This might indicate that there are
strong multicollinearity or other numerical problems.

ols_df_afford = all_i[all_i['Price per bedroom per month']<2000][['Price per bedroom per month','No. bedrooms','Distance from LSE','Crime No.','LSE accommodation']]
ols_df_afford['LSE accommodation'] = ols_df_afford['LSE accommodation'].astype('int')
ols_df_afford = ols_df_afford.rename(columns={'Price per bedroom per month':'price_per_bedroom_per_month','No. bedrooms':'no_bedrooms','Distance from LSE':'distance_from_LSE','Crime No.':'crime_no','LSE accommodation':'LSE_accommodation'})

result = sm.ols(formula='price_per_bedroom_per_month ~ no_bedrooms + distance_from_LSE + LSE_accommodation', 
                data=ols_df_afford).fit(cov_type='HC0') # Use robust standard errors
print(result.summary())

                                 OLS Regression Results                                
=======================================================================================
Dep. Variable:     price_per_bedroom_per_month   R-squared:                       0.349
Model:                                     OLS   Adj. R-squared:                  0.345
Method:                          Least Squares   F-statistic:                     115.5
Date:                         Thu, 02 May 2024   Prob (F-statistic):           7.99e-57
Time:                                 10:33:16   Log-Likelihood:                -3525.1
No. Observations:                          503   AIC:                             7058.
Df Residuals:                              499   BIC:                             7075.
Df Model:                                    3                                         
Covariance Type:                           HC0                                         
=====================================================================================
                        coef    std err          z      P>|z|      [0.025      0.975]
-------------------------------------------------------------------------------------
Intercept          1890.9553     42.478     44.516      0.000    1807.701    1974.210
no_bedrooms        -155.6817      9.788    -15.905      0.000    -174.866    -136.497
distance_from_LSE   -56.7037     13.270     -4.273      0.000     -82.712     -30.696
LSE_accommodation  -469.2789     32.730    -14.338      0.000    -533.429    -405.129
==============================================================================
Omnibus:                        0.936   Durbin-Watson:                   1.815
Prob(Omnibus):                  0.626   Jarque-Bera (JB):                1.020
Skew:                           0.060   Prob(JB):                        0.600
Kurtosis:                       2.815   Cond. No.                         29.6
==============================================================================

Notes:
[1] Standard Errors are heteroscedasticity robust (HC0)

ols_df_afford_i = all_i[all_i['Price per bedroom per month']<2000][['Price per bedroom per month','No. bedrooms','Distance from LSE','Crime No.','LSE accommodation']]
ols_df_afford_i['LSE accommodation'] = ols_df_afford_i['LSE accommodation'].astype('int')
ols_df_afford_i['interaction'] = ols_df_afford_i['LSE accommodation'] * ols_df_afford_i['Price per bedroom per month']
ols_df_afford_i = ols_df_afford_i.rename(columns={'Price per bedroom per month':'price_per_bedroom_per_month','No. bedrooms':'no_bedrooms','Distance from LSE':'distance_from_LSE','Crime No.':'crime_no','LSE accommodation':'LSE_accommodation'})

result_i = sm.ols(formula='price_per_bedroom_per_month ~ distance_from_LSE + LSE_accommodation + interaction', 
                  data=ols_df_afford_i).fit(cov_type='HC0') # Use robust standard errors
print(result_i.summary())

                                 OLS Regression Results                                
=======================================================================================
Dep. Variable:     price_per_bedroom_per_month   R-squared:                       0.039
Model:                                     OLS   Adj. R-squared:                  0.033
Method:                          Least Squares   F-statistic:                     43.81
Date:                         Thu, 02 May 2024   Prob (F-statistic):           3.81e-25
Time:                                 10:33:21   Log-Likelihood:                -3623.0
No. Observations:                          503   AIC:                             7254.
Df Residuals:                              499   BIC:                             7271.
Df Model:                                    3                                         
Covariance Type:                           HC0                                         
=====================================================================================
                        coef    std err          z      P>|z|      [0.025      0.975]
-------------------------------------------------------------------------------------
Intercept          1550.7842     50.403     30.768      0.000    1451.997    1649.572
distance_from_LSE   -73.0227     16.969     -4.303      0.000    -106.281     -39.765
LSE_accommodation -1075.0654    156.209     -6.882      0.000   -1381.230    -768.901
interaction           0.6895      0.135      5.093      0.000       0.424       0.955
==============================================================================
Omnibus:                       32.996   Durbin-Watson:                   1.731
Prob(Omnibus):                  0.000   Jarque-Bera (JB):               16.228
Skew:                           0.247   Prob(JB):                     0.000299
Kurtosis:                       2.272   Cond. No.                     1.58e+04
==============================================================================

Notes:
[1] Standard Errors are heteroscedasticity robust (HC0)
[2] The condition number is large, 1.58e+04. This might indicate that there are
strong multicollinearity or other numerical problems.

import plotly.express as pltx

afford_lse_accomm = all_i[all_i['Price per bedroom per month']<2000][all_i['LSE accommodation']]
afford_priv_prop = all_i[all_i['Price per bedroom per month']<2000][all_i['LSE accommodation'] == False]

graph = pltx.scatter(all_i[all_i['Price per bedroom per month']<2000], x = "Distance from LSE", y = "Price per bedroom per month", 
                     trendline='ols', hover_data=["Name", "Outcode"], color="LSE accommodation",
                     title = "<b>Comparison between prices of privately rented property and LSE accommodation per month</b>")

# Find the slope of the ols line of best fit and add annotation:
slope, _ = np.polyfit(afford_priv_prop["Distance from LSE"], afford_priv_prop["Price per bedroom per month"], 1)

graph.add_annotation(x = 0.7, y = 1500, text=f"<b>Slope = {slope:.2f}</b>", showarrow=True, arrowhead=2, ax = -20, ay = -80, font = dict(size = 12))
graph.add_shape(type="line", x0=0.4, y0=1200, x1=1.5, y1=1200, line=dict(color="black", width=1))
graph.add_shape(type="line", x0=1.5, y0=1200, x1=1.5, y1=590, line=dict(color="black", width=1))

graph.update_layout(title_x = 0.5) # Center the title

graph.show('notebook')

plt.figure(figsize=(12, 8))
ax = sns.scatterplot(data = all_i, x = 'Distance from LSE', y='Price per bedroom per month',
                hue = 'LSE accommodation',style = 'LSE accommodation',size = 'LSE accommodation',sizes = (250,50))
ax.set_xlabel("Distance from LSE (KM)", fontsize = 12)
ax.set_ylabel("Price per bedroom per month (£)", fontsize = 12)
ax.set_title("Comparison between prices of privately rented property and LSE accommodation per month", fontsize = 14, fontweight = 'bold')
ax.grid(axis='y', linestyle='--', alpha=0.5)
ax.plot()

[]

plt.figure(figsize=(12, 8))
ax = sns.scatterplot(data = all_i, x = 'Distance from LSE', y='Price per bedroom per year',
                hue = 'LSE accommodation',style = 'LSE accommodation',size = 'LSE accommodation',sizes = (250,50))
ax.set_xlabel("Distance from LSE (KM)", fontsize = 12)
ax.set_ylabel("Price per bedroom per year (£)", fontsize = 12)
ax.set_title("Comparison between prices of privately rented property and LSE accommodation per year", fontsize = 14, fontweight = 'bold')
ax.grid(axis='y', linestyle='--', alpha=0.5)
ax.plot()

[]

	hall	(avg) cost per week	contract length	(avg) contract cost	distance from LSE	crimes number	outcode
0	urbanest Westminster Bridge	297.995	39	11621.805	1.5	3219	SE1
1	College Hall	289.730	40	11589.200	1.2	7655	WC1E
2	International Hall	266.280	40	10651.200	1.0	6247	WC1N
3	Bankside House	259.700	39	10128.300	1.5	2392	SE1
4	Carr-Saunders Hall	257.250	31	7974.750	1.6	7544	W1T
5	Connaught Hall	273.630	40	10945.200	1.3	6760	WC1H
6	High Holborn Residence	317.800	39	12394.200	0.5	7225	WC1V
7	Passfield Hall	252.875	31	7839.125	1.5	6240	WC1H
8	Nutford House	250.180	40	10007.200	3.1	2803	W1H
9	Rosebery Hall	255.850	39	9978.150	1.6	2646	EC1R
10	The Garden Halls	279.195	40	11167.800	1.6	4778	WC1H

	Name	Outcode	Price per month	Price per bedroom per month	Price per bedroom per year	No. bedrooms	Distance from LSE	Crime No.	LSE accommodation
751	1 bed flat	SE5	3990	3990.0	47880.0	1	4.0	1388	False
752	2 bed flat	SW11	4000	2000.0	24000.0	2	4.0	1124	False
753	urbanest Westminster Bridge	SE1	1291	1291.0	11621.0	1	1.5	3219	True
754	College Hall	WC1E	1255	1255.0	11589.0	1	1.2	7655	True
755	International Hall	WC1N	1153	1153.0	10651.0	1	1.0	6247	True
756	Bankside House	SE1	1125	1125.0	10128.0	1	1.5	2392	True
757	Carr-Saunders Hall	W1T	1114	1114.0	7974.0	1	1.6	7544	True
758	Connaught Hall	WC1H	1185	1185.0	10945.0	1	1.3	6760	True
759	High Holborn Residence	WC1V	1377	1377.0	12394.0	1	0.5	7225	True
760	Passfield Hall	WC1H	1095	1095.0	7839.0	1	1.5	6240	True
761	Nutford House	W1H	1084	1084.0	10007.0	1	3.1	2803	True
762	Rosebery Hall	EC1R	1108	1108.0	9978.0	1	1.6	2646	True
763	The Garden Halls	WC1H	1209	1209.0	11167.0	1	1.6	4778	True

	Name	Outcode	Price per month	Price per bedroom per month	Price per bedroom per year	No. bedrooms	Distance from LSE	Crime No.	LSE accommodation
0	3 bed flat	WC2A	7800	2600.0	31200.0	3	0.17	5678	False
1	studio flat	WC2B	2396	2396.0	28752.0	1	0.31	6949	False
2	2 bed flat	WC2R	5633	2816.5	33798.0	2	0.35	5585	False
3	1 bed flat	WC2A	2500	2500.0	30000.0	1	0.40	5134	False
4	1 bed flat	EC4A	2600	2600.0	31200.0	1	0.48	4727	False
...	...	...	...	...	...	...	...	...	...
759	High Holborn Residence	WC1V	1377	1377.0	12394.0	1	0.50	7225	True
760	Passfield Hall	WC1H	1095	1095.0	7839.0	1	1.50	6240	True
761	Nutford House	W1H	1084	1084.0	10007.0	1	3.10	2803	True
762	Rosebery Hall	EC1R	1108	1108.0	9978.0	1	1.60	2646	True
763	The Garden Halls	WC1H	1209	1209.0	11167.0	1	1.60	4778	True

	Name	Outcode	Price per month	Price per bedroom per month	Price per bedroom per year	No. bedrooms	Distance from LSE	Crime No.	LSE accommodation
count	764	764	764.00	764.00	764.00	764.00	764.00	764.00	764
unique	35	68	NaN	NaN	NaN	NaN	NaN	NaN	2
top	2 bed flat	NW1	NaN	NaN	NaN	NaN	NaN	NaN	False
freq	198	93	NaN	NaN	NaN	NaN	NaN	NaN	753
mean	NaN	NaN	2638.11	1814.00	21713.52	2.09	2.80	2639.82	NaN
std	NaN	NaN	1436.35	856.71	10331.14	1.22	0.87	1493.49	NaN
min	NaN	NaN	650.00	650.00	7800.00	1.00	0.17	1124.00	NaN
25%	NaN	NaN	1500.00	1200.00	14400.00	1.00	2.19	1764.00	NaN
50%	NaN	NaN	2449.00	1586.17	19034.00	2.00	3.03	2249.00	NaN
75%	NaN	NaN	3300.50	2275.00	27300.00	3.00	3.48	2631.25	NaN
max	NaN	NaN	8333.00	8333.00	99996.00	7.00	4.00	7873.00	NaN

	Price per month	Price per bedroom per month	Price per bedroom per year	No. bedrooms	Distance from LSE	Crime No.
count	762.00	762.00	762.00	762.00	762.00	762.00
mean	2642.15	1815.88	21735.93	2.07	2.80	2641.45
std	1436.02	856.99	10334.70	1.19	0.87	1494.97
min	650.00	650.00	7800.00	1.00	0.17	1124.00
25%	1517.75	1200.00	14400.00	1.00	2.19	1765.00
50%	2450.00	1589.50	19074.00	2.00	3.03	2249.00
75%	3301.50	2275.00	27300.00	3.00	3.48	2631.75
max	8333.00	8333.00	99996.00	6.00	4.00	7873.00

Analysis of affordability and safety of housing close to LSE

Contents¶

1. Introduction¶

1.1. Motivation¶

1.2. Research Questions¶

1.3. Originality¶

1.4. Overview of Method¶

2. Data Acquisition¶

2.1. OpenRent General Webpage Scraping¶

2.2. OpenRent Individual Listing Webpage Scraping¶

2.3. Adding Crime Information from Police.uk API¶

2.4. Scraping Data for LSE Halls¶

3. Prepare the Data for Analysis¶

3.1. Adding Private Property Data to a DataFrame¶

3.2. Adding LSE Accommodations to DataFrame¶

4. Data Analysis¶

4.1. Summary Statistics and Final Data Corrections¶

4.2. General Correlation¶

4.3. Effects of no. bedrooms on cost per bedroom¶

4.4. Number of affordable properties and safety by postcode¶

4.5. Effects of distance from LSE on price¶

4.5.1. Regression analysis¶

4.5.2. Regression results¶

4.5.3. Further regression evaluation¶

4.5.4. Regression excluding crime numbers and properties over £2000¶

4.6. Comparison to LSE accommodation¶

4.6.1. Regression Analysis and Results¶

4.6.2. Visualisation of regressed dataset¶

4.6.3. Visualisation of the whole dataset¶

5. Conclusion¶

5.1. Summary of Results¶

5.2. Interpretation of Results¶

5.3. Limitations¶

5.4. Potential Further Analysis¶

6. References¶

	index	Outcode	Number of Available Properties below £1400 per Bedroom	Avg Crime No. in Outcode
0	14	NW1	44	2341.62
1	11	N1	43	2194.99
2	16	SE1	33	2282.69
3	0	E1	28	2361.83
4	22	SW1V	21	1530.25

	Price per bedroom per month
	count	mean	std	min	25%	50%	75%	max
No. bedrooms
1	309.0	2347.18	932.79	700.0	1795.0	2250.0	2730.00	8333.00
2	239.0	1684.15	594.82	866.5	1300.0	1500.0	1897.75	4166.50
3	112.0	1290.78	422.48	700.0	1000.0	1200.0	1433.33	2777.67
4	63.0	1058.94	264.39	650.0	862.5	1000.0	1150.00	2000.00
5	30.0	1224.33	446.09	715.0	997.0	1150.0	1267.25	2383.00
6	9.0	877.81	45.36	860.0	860.0	860.0	860.00	997.00